const puppeteer = require('puppeteer');
const fs = require('fs');

let totalBlogs = [];

async function main() {
  const MAX_CNT = 300;
  const SLEEP_TIME = 100;

  const tab = 'segmentfault';
  const sub_tab = 'backend';
  const url = `https://segmentfault.com/channel/${sub_tab}`;
  const fileName = `segmentfault_${sub_tab}.json`;


  const browser = await puppeteer.launch({headless: false, slowMo: 0});
  const page = (await browser.pages())[0];
  await page.setViewport({width: 1500, height: 0});
  page.on('console', msg => {
    console.log(msg.text());
  });

  page.on('pageerror', msg => {
    console.log(msg);
  });


  await page.exposeFunction('dateDiff', (nowDate, diffString) => {
    function _fmtDate(obj, needTime) {
      var date = new Date(obj);
      var y = 1900 + date.getYear();
      var m = "0" + (date.getMonth() + 1);
      var d = "0" + date.getDate();
      var h = "0" + date.getHours();
      var min = "0" + date.getMinutes();
      var s = "0" + date.getSeconds();
      if (needTime) {
        return y + "-" + m.substring(m.length - 2, m.length) + "-" + d.substring(d.length - 2, d.length) +
          ' ' + h.substring(h.length - 2, h.length) + ":00:00";
      } else {
        return y + "-" + m.substring(m.length - 2, m.length) + "-" + d.substring(d.length - 2, d.length) + ' 00:00:00';
      }
    }

    function _dateDiff(nowDate, diffString) {
      nowDate = new Date(nowDate);

      const minuteReg = /分钟前/;
      const hourReg = /小时前/;
      const dayReg = /天前/;
      const monthReg = /月前/;
      const yearReg = /年前/;
      const yyyymmddReg = /年.+月.+日/;
      const mmddReg = /^\d+月.+日/;
      if (minuteReg.test(diffString)) {
        return _fmtDate(nowDate.getTime(), true);
      }

      if (hourReg.test(diffString)) {
        const hourTimestamp = Number(diffString.split('小时前')[0]) * 3600 * 1000;
        return _fmtDate(nowDate.getTime() - hourTimestamp, true);
      }
      if (dayReg.test(diffString)) {
        const dayTimestamp = Number(diffString.split('天前')[0]) * 24 * 3600 * 1000;
        return _fmtDate(nowDate.getTime() - dayTimestamp, false);
      }
      if (monthReg.test(diffString)) {
        const monthTimestamp = Number(diffString.split('月前')[0]) * 30 * 24 * 3600 * 1000;
        return _fmtDate(nowDate.getTime() - monthTimestamp, false);
      }
      if (yearReg.test(diffString)) {
        const yearTimestamp = Number(diffString.split('年前')[0]) * 365 * 24 * 3600 * 1000;
        return _fmtDate(nowDate.getTime() - yearTimestamp, false);
      }
      if (yyyymmddReg.test(diffString)) {
        return _fmtDate(new Date(Date.parse(diffString.replace('年', '-').replace('月', '-').replace('日', ''))), false);
      }

      if (mmddReg.test(diffString)) {
        let yearStr = (new Date()).getFullYear() + '年';
        return _fmtDate(new Date(Date.parse((yearStr + diffString).replace('年', '-').replace('月', '-').replace('日', ''))), false);
      }

      return '时间格式有误';
    }

    return _dateDiff(nowDate, diffString);
  });


  await page.goto(url);
  await page.waitFor('body > div.wrapper > div.container.mt30 > div > div.col-md-7.middle > div.news-list > div');


  let preScrollHeight = 0;
  let scrollHeight = -1;
  let cnt = 0;
  let retry_cnt = 0;
  while (cnt < MAX_CNT) {
    // 详情信息是根据滚动异步加载，所以需要让页面滚动到屏幕最下方，通过延迟等待的方式进行多次滚动
    let scrollH1 = await page.evaluate(async () => {
      let h1 = document.body.scrollHeight;
      window.scrollTo(0, h1);
      return h1;
    });

    // 延迟等待时间
    await page.waitFor(SLEEP_TIME);

    let scrollH2 = await page.evaluate(async () => {
      return document.body.scrollHeight;
    });
    let scrollResult = [scrollH1, scrollH2];
    preScrollHeight = scrollResult[0];
    scrollHeight = scrollResult[1];

    if (preScrollHeight == scrollHeight) {
      retry_cnt++;
      if (retry_cnt > 100) {
        break;
      }
      try {
        await page.click('#btn-load-more > a');
      } catch (e) {
        console.log('try click error...');
      }
    } else {
      retry_cnt = 0;
      cnt++;
    }
    console.log(scrollResult, cnt);
  }

  console.log('before last totalBlogs length: ' + totalBlogs.length);
  let blogs = await collectBlogs(page, tab, sub_tab);
  console.log('last blogs length: ' + blogs.length);
  totalBlogs = totalBlogs.concat(blogs);
  console.log('totalBlogs length: ' + totalBlogs.length);

  let writerStream = fs.createWriteStream(`crawler/crawler_data/${fileName}`);
  writerStream.write(JSON.stringify(totalBlogs));
  console.log('write done');


  // browser.close();
};

async function collectBlogs(page, tab, subTab) {
  let blogs = await page.$$eval('body > div.wrapper > div.container.mt30 > div > div.col-md-7.middle > div.news-list > div', async (eles, tab, subTab) => {
    console.log('eles length: ', eles.length);
    let tmpBlogs = [];
    eles.forEach(async ele => {
      let old_date = ele.querySelector('div > div > span.author.pr20').innerText.split('·')[1];
      let blog_date = await window.dateDiff(new Date(), old_date);
      let title = ele.querySelector('div > a:last-of-type > div.mb5.mt5 > h4').innerText;
      let content_url = ele.querySelector('div > a:last-of-type').href;
      let content_txt = ele.querySelector('div > a:last-of-type > div.article-excerpt').innerText;
      tmpBlogs.push({
        title: title,
        content: `[${content_txt}](${content_url})`,
        blog_date: blog_date,
        tab: tab,
        sub_tab: subTab
      });
    });
    return tmpBlogs;
  }, tab, subTab);
  return blogs;
}

main();


